{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "## Exploring the SciQ dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {},
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# provide your own OpenAI API key here\n",
    "os.environ['OPENAI_API_KEY'] = 'sk-xxxxxxxxxxxxxxxxx'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {},
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bc181e31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: refuel-autolabel[openai] in /opt/homebrew/lib/python3.10/site-packages (0.0.1)\n",
      "Requirement already satisfied: loguru>=0.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.5.3)\n",
      "Requirement already satisfied: numpy>=1.23.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.23.3)\n",
      "Requirement already satisfied: requests>=2.27.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.28.1)\n",
      "Requirement already satisfied: datasets>=2.7.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.11.0)\n",
      "Requirement already satisfied: langchain>=0.0.190 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.0.191)\n",
      "Requirement already satisfied: nervaluate>=0.1.8 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.1.8)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.5.0)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.1.2)\n",
      "Requirement already satisfied: tenacity>=8.2.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (8.2.2)\n",
      "Requirement already satisfied: SQLAlchemy==1.4.47 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.4.47)\n",
      "Requirement already satisfied: regex>=2023.6.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2023.6.3)\n",
      "Requirement already satisfied: rich>=13.3.5 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (13.4.1)\n",
      "Requirement already satisfied: scipy>=1.10.1 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.1)\n",
      "Requirement already satisfied: pydantic>=1.10.9 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.9)\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.13.0)\n",
      "Requirement already satisfied: matplotlib>=3.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.6.0)\n",
      "Requirement already satisfied: wget>=3.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.2)\n",
      "Requirement already satisfied: openai>=0.27.4 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.27.6)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.3.3)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (9.0.0)\n",
      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.3.5.1)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (4.64.1)\n",
      "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.70.13)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (2022.10.0)\n",
      "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.8.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.14.1)\n",
      "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (21.3)\n",
      "Requirement already satisfied: responses<0.19 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.18.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (6.0)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (0.5.7)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (2.8.4)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (1.2.4)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (4.37.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.10/site-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2022.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/homebrew/lib/python3.10/site-packages (from pydantic>=1.10.9->refuel-autolabel[openai]) (4.3.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2022.9.14)\n",
      "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.15.1)\n",
      "Requirement already satisfied: joblib>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (22.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.8.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.3.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.2.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (3.19.0)\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (1.5.1)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.8.0)\n",
      "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets>=2.7.0->refuel-autolabel[openai]) (3.8.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/homebrew/lib/python3.10/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=13.3.5->refuel-autolabel[openai]) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->refuel-autolabel[openai]) (1.12.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.4.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0ced632",
   "metadata": {},
   "source": [
    "#### Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1bdeefe1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading seed example dataset to \"seed.csv\"...\n",
      "100% [..........................................................] 29687 / 29687\n",
      "\n",
      "Downloading test dataset to \"test.csv\"...\n",
      "100% [........................................................] 119128 / 119128"
     ]
    }
   ],
   "source": [
    "from autolabel import get_data\n",
    "\n",
    "get_data('sciq')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16f17787",
   "metadata": {},
   "source": [
    "This downloads two datasets:\n",
    "* `test.csv`: This is the larger dataset we are trying to label using LLMs\n",
    "* `seed.csv`: This is a small dataset where we already have human-provided labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {},
   "source": [
    "## Start the labeling process!\n",
    "\n",
    "Labeling with Autolabel is a 3-step process:\n",
    "* First, we specify a labeling configuration (see `config.json` below)\n",
    "* Next, we do a dry-run on our dataset using the LLM specified in `config.json` by running `agent.plan`\n",
    "* Finally, we run the labeling with `agent.run`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the config\n",
    "with open('config_sciq.json', 'r') as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0983618",
   "metadata": {},
   "source": [
    "Let's review the configuration file below. You'll notice the following useful keys:\n",
    "* `task_type`: `question_answering` (since it's a question answering task)\n",
    "* `model`: `{'provider': 'openai', 'name': 'gpt-3.5-turbo'}` (use a specific OpenAI model)\n",
    "* `prompt.task_guidelines`: `'You are an expert at answer science questions...` (how we describe the task to the LLM)\n",
    "* `prompt.few_shot_num`: 10 (how many labeled examples to provide to the LLM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "08a1f895",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task_name': 'ScienceQuestionAnswering',\n",
       " 'task_type': 'question_answering',\n",
       " 'dataset': {'label_column': 'answer', 'delimiter': ','},\n",
       " 'model': {'provider': 'openai', 'name': 'gpt-3.5-turbo'},\n",
       " 'prompt': {'task_guidelines': 'You are an expert at answer science questions. Your job is to answer the given question, using the options provided for each question. Choose the best answer for the question from among the options provided',\n",
       "  'example_template': 'Question: {question}\\nOptions: {options}\\nAnswer: {answer}',\n",
       "  'few_shot_examples': 'seed.csv',\n",
       "  'few_shot_selection': 'semantic_similarity',\n",
       "  'few_shot_num': 10}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "acb4a3de-fa84-4b94-b17a-7a6fac892a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an agent for labeling\n",
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $2.4461 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 800     </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0031 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$2.4461\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m800    \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0031\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at answer science questions. Your job is to answer the given question, using the options provided for each question. Choose the best answer for the question from among the options provided\n",
      "\n",
      "You will return the answer one element: \"the correct label\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Question: What do you call health-promoting molecules that inhibit the oxidation of other molecules?\n",
      "Options: ['nutrients', 'antioxidants', 'neurotransmitters', 'hormones']\n",
      "Answer: antioxidants\n",
      "\n",
      "Question: Highly reactive nonmetals, which only accept electrons and do not give them up, make poor what?\n",
      "Options: ['electricity conductors', 'insulators', 'electromagnets', 'alloys']\n",
      "Answer: electricity conductors\n",
      "\n",
      "Question: The majority of elements, including iron and copper, are of what type?\n",
      "Options: ['oils', 'metals', 'minerals', 'acids']\n",
      "Answer: metals\n",
      "\n",
      "Question: A hydrogen atom with one neutron is called what?\n",
      "Options: ['magnesium', 'deuterium', 'covalent', 'ionic']\n",
      "Answer: deuterium\n",
      "\n",
      "Question: A fuel cell is a galvanic cell that requires a constant external supply of what?\n",
      "Options: ['reactants', 'complexes', 'generators', 'electricity']\n",
      "Answer: reactants\n",
      "\n",
      "Question: What is the third class of elements after metals and nonmetals?\n",
      "Options: ['metalloids', 'halogens', 'synthetics', 'noble gases']\n",
      "Answer: metalloids\n",
      "\n",
      "Question: What compounds form crystals instead of molecules?\n",
      "Options: ['magnetic compounds', 'soluble compounds', 'metallic compounds', 'ionic compounds']\n",
      "Answer: ionic compounds\n",
      "\n",
      "Question: Solid carbon dioxide is called what?\n",
      "Options: ['carbohydrate', 'dry ice', 'blue ice', 'liquid nitrogen']\n",
      "Answer: dry ice\n",
      "\n",
      "Question: The name of a simple covalent compound can be determined from its chemical this?\n",
      "Options: ['formula', 'models', 'map', 'configuration']\n",
      "Answer: formula\n",
      "\n",
      "Question: What acids are the structural components of many lipids and may be saturated or unsaturated?\n",
      "Options: ['fatty acids', 'carbonic acids', 'fundamental acids', 'ionic acids']\n",
      "Answer: fatty acids\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Question: Compounds that are capable of accepting electrons, such as o 2 or f2, are called what?\n",
      "Options: ['Oxygen', 'antioxidants', 'residues', 'oxidants']\n",
      "Answer: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "ds = AutolabelDataset(\"data/sciq/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Cost: 0.1063\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> f1   </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└──────┴─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mf1  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.95\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└──────┴─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af8d5d58",
   "metadata": {},
   "source": [
    "We are at 94% accuracy when labeling the first 100 examples. Let's see if we can use confidence scores to improve accuracy further by removing the less confident examples from our labeled set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d7645ab",
   "metadata": {},
   "source": [
    "### Compute confidence scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f13a18d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start computing confidence scores (using Refuel's LLMs)\n",
    "os.environ['REFUEL_API_KEY'] = 'xxxxxxxxxxxxxxxxx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5fbc1264",
   "metadata": {},
   "outputs": [],
   "source": [
    "config[\"model\"][\"compute_confidence\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1998f5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "119e6f22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $2.4463 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 800     </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0031 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$2.4463\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m800    \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0031\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at answer science questions. Your job is to answer the given question, using the options provided for each question. Choose the best answer for the question from among the options provided\n",
      "\n",
      "You will return the answer one element: \"the correct label\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Question: What do you call health-promoting molecules that inhibit the oxidation of other molecules?\n",
      "Options: ['nutrients', 'antioxidants', 'neurotransmitters', 'hormones']\n",
      "Answer: antioxidants\n",
      "\n",
      "Question: Highly reactive nonmetals, which only accept electrons and do not give them up, make poor what?\n",
      "Options: ['electricity conductors', 'insulators', 'electromagnets', 'alloys']\n",
      "Answer: electricity conductors\n",
      "\n",
      "Question: The majority of elements, including iron and copper, are of what type?\n",
      "Options: ['oils', 'metals', 'minerals', 'acids']\n",
      "Answer: metals\n",
      "\n",
      "Question: A hydrogen atom with one neutron is called what?\n",
      "Options: ['magnesium', 'deuterium', 'covalent', 'ionic']\n",
      "Answer: deuterium\n",
      "\n",
      "Question: A fuel cell is a galvanic cell that requires a constant external supply of what?\n",
      "Options: ['reactants', 'complexes', 'generators', 'electricity']\n",
      "Answer: reactants\n",
      "\n",
      "Question: What is the third class of elements after metals and nonmetals?\n",
      "Options: ['metalloids', 'halogens', 'synthetics', 'noble gases']\n",
      "Answer: metalloids\n",
      "\n",
      "Question: What compounds form crystals instead of molecules?\n",
      "Options: ['magnetic compounds', 'soluble compounds', 'metallic compounds', 'ionic compounds']\n",
      "Answer: ionic compounds\n",
      "\n",
      "Question: Solid carbon dioxide is called what?\n",
      "Options: ['carbohydrate', 'dry ice', 'blue ice', 'liquid nitrogen']\n",
      "Answer: dry ice\n",
      "\n",
      "Question: The name of a simple covalent compound can be determined from its chemical this?\n",
      "Options: ['formula', 'models', 'map', 'configuration']\n",
      "Answer: formula\n",
      "\n",
      "Question: What acids are the structural components of many lipids and may be saturated or unsaturated?\n",
      "Options: ['fatty acids', 'carbonic acids', 'fundamental acids', 'ionic acids']\n",
      "Answer: fatty acids\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Question: Compounds that are capable of accepting electrons, such as o 2 or f2, are called what?\n",
      "Options: ['Oxygen', 'antioxidants', 'residues', 'oxidants']\n",
      "Answer: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "ds = AutolabelDataset(\"data/sciq/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "63c74705",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 15:09:14 autolabel.labeler INFO: Task run already exists.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">There is an existing task with following details: <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'866618266'</span> <span style=\"color: #808000; text-decoration-color: #808000\">created_at</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">datetime</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">.datetime</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2023</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, \n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">26</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">243688</span><span style=\"font-weight: bold\">)</span> <span style=\"color: #808000; text-decoration-color: #808000\">task_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'ec731deb33e8af15075d7e485230494b'</span> <span style=\"color: #808000; text-decoration-color: #808000\">dataset_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'62edd46821e02ae779b2566815a84de9'</span> \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">current_index</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span> <span style=\"color: #808000; text-decoration-color: #808000\">output_file</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'test_labeled.csv'</span> <span style=\"color: #808000; text-decoration-color: #808000\">status</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">TaskStatus.ACTIVE:</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'active'</span><span style=\"font-weight: bold\">&gt;</span> <span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span> <span style=\"color: #808000; text-decoration-color: #808000\">metrics</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "There is an existing task with following details: \u001b[33mid\u001b[0m=\u001b[32m'866618266'\u001b[0m \u001b[33mcreated_at\u001b[0m=\u001b[1;35mdatetime\u001b[0m\u001b[1;35m.datetime\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m2023\u001b[0m, \u001b[1;36m6\u001b[0m, \u001b[1;36m14\u001b[0m, \u001b[1;36m15\u001b[0m, \u001b[1;36m4\u001b[0m, \n",
       "\u001b[1;36m26\u001b[0m, \u001b[1;36m243688\u001b[0m\u001b[1m)\u001b[0m \u001b[33mtask_id\u001b[0m=\u001b[32m'ec731deb33e8af15075d7e485230494b'\u001b[0m \u001b[33mdataset_id\u001b[0m=\u001b[32m'62edd46821e02ae779b2566815a84de9'\u001b[0m \n",
       "\u001b[33mcurrent_index\u001b[0m=\u001b[1;36m100\u001b[0m \u001b[33moutput_file\u001b[0m=\u001b[32m'test_labeled.csv'\u001b[0m \u001b[33mstatus\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mTaskStatus.ACTIVE:\u001b[0m\u001b[39m \u001b[0m\u001b[32m'active'\u001b[0m\u001b[1m>\u001b[0m \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m \u001b[33mmetrics\u001b[0m=\u001b[3;35mNone\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluating the existing task<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluating the existing task\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 0.5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> f1   </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└──────┴─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mf1  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.95\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.95\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└──────┴─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span> examples have been labeled so far.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m100\u001b[0m examples have been labeled so far.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────── </span>Last Annotated Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────── \u001b[0mLast Annotated Example\u001b[92m ──────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Prompt</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mPrompt\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at answer science questions. Your job is to answer the given question, using the options provided for each question. Choose the best answer for the question from among the options provided\n",
      "\n",
      "You will return the answer one element: \"the correct label\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Question: The male gametophyte releases what, which swim - propelled by their flagella - to reach and fertilize the female gamete or egg?\n",
      "Options: ['sperm', 'cytoplasm', 'tadpoles', 'dna']\n",
      "Answer: sperm\n",
      "\n",
      "Question: Prophase is preceded by a preprophase stage in what type of cells?\n",
      "Options: ['hair and nail cells', 'plant cells', 'egg cells', 'brain cells']\n",
      "Answer: plant cells\n",
      "\n",
      "Question: What form of reproduction creates offspring that are genetically identical to the parent?\n",
      "Options: ['microscopic', 'primitive', 'sexual', 'asexual']\n",
      "Answer: asexual\n",
      "\n",
      "Question: In humans, fertilization occurs soon after the oocyte leaves this?\n",
      "Options: ['placenta', 'testes', 'ovary', 'egg']\n",
      "Answer: ovary\n",
      "\n",
      "Question: What term describes reproducing by external fertilization after the female sheds large numbers of small eggs?\n",
      "Options: ['homozygous', 'oviparous', 'glandular', 'glaucous']\n",
      "Answer: oviparous\n",
      "\n",
      "Question: What is an individual with more than the correct number of chromosome sets called?\n",
      "Options: ['polyploid', 'autotrophs', 'diploid', 'autosome']\n",
      "Answer: polyploid\n",
      "\n",
      "Question: Above the meristem, the rest of the root is covered with a single layer of what type of cells, which may have root hairs?\n",
      "Options: ['single celled', 'vegetative', 'dermal', 'epidermal']\n",
      "Answer: epidermal\n",
      "\n",
      "Question: What takes place where septa is found?\n",
      "Options: ['forming of brain cells', 'forming of muscle tissue', 'forming of reproductive cells', 'forming of stem cells']\n",
      "Answer: forming of reproductive cells\n",
      "\n",
      "Question: The bilaterians are divided into deuterostomes and what else?\n",
      "Options: ['gymnosperms', 'progestins', 'protostomes', 'protozoa']\n",
      "Answer: protostomes\n",
      "\n",
      "Question: Vivipary refers to the development and nourishment of a what?\n",
      "Options: ['embryo', 'seed', 'egg', 'nucleus']\n",
      "Answer: embryo\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Question: In humans, the only haploid cells are what reproductive cells?\n",
      "Options: ['dna and egg', 'sperm and dna', 'sperm and egg', 'uteral and sperm']\n",
      "Answer: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">Annotation</span>: </pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mAnnotation\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sperm and egg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Do you want to resume the task? <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">[y/n]</span>: </pre>\n"
      ],
      "text/plain": [
       "Do you want to resume the task? \u001b[1;35m[y/n]\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Deleted the existing task and starting a new one<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Deleted the existing task and starting a new one\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 15:22:10 langchain.embeddings.openai WARNING: Retrying langchain.embeddings.openai.embed_with_retry.<locals>._embed_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 0.6986\n",
      "Actual Cost: 0.0268\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> f1     </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9999    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.01            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 30      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9982    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.3             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9375 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 32      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9978    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9375   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.32            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.973  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 74      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9036    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.973    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.74            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9644 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 75      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8965    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.96     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.75            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.969  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 86      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7238    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9651   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.86            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9655 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 87      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7105    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.954    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.87            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9674 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 92      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6058    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9565   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.92            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.957  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 93      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5412    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9462   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.93            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9583 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 96      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9479   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.96            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9485 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 97      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.475     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9381   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.97            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0376    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└────────┴─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mf1    \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.95  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1.0   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9999   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.01           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1.0   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m30     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9982   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.3            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9375\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m32     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9978   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9375  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.32           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.973 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m74     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9036   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.973   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.74           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9644\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m75     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8965   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.96    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.75           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.969 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m86     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7238   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9651  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.86           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9655\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m87     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7105   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.954   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.87           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9674\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m92     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6058   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9565  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.92           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.957 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m93     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5412   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9462  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.93           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9583\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m96     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9479  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.96           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9485\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m97     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.475    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9381  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.97           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.95  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0376   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└────────┴─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eede651",
   "metadata": {},
   "source": [
    "Looking at the table above, we can see that if we set the confidence threshold at `0.9036`, we are able to label at 97.3% accuracy and getting a completion rate of 74%. This means, we would ignore all the data points where confidence score is less than `0.9036` (which would end up being around 26% of all samples). This would, however, guarantee a very high quality labeled dataset for us. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
